#coding:utf-8
import requests
from lxml import etree
import time
import itertools
import re
import wget
import os
import sys 
reload(sys) 
sys.setdefaultencoding( "utf-8" ) 

from fake_useragent import UserAgent
ua = UserAgent()

class MaiZi(object):
	def __init__(self):
		self.url = ''
		self.html = ''
		self.info = ''
		self.path = ''
	#获取课程列表
	def Course_List(self,html):
		tree = etree.HTML(html)
		course_list = tree.xpath('/html/body/div[3]/div/div[3]/ul/li')
		print len(course_list)
		course = []
		for i in range(len(course_list)):
			name = course_list[i].xpath('a/@title')
			print "序号：%d \n课程名：%s"%(i+1,name[0])
			href = course_list[i].xpath('a/@href')
			course.append('http://www.maiziedu.com' + href[0])
		return course
	#对课程页面进一步处理，集成函数
	def Change_Box(self,course_list):
		#遍历所有课程详情页链接
		for j in range(1,len(course_list)+1):
			time.sleep(3)
			#课程详情页URL链接
			print course_list[j-1]
			#课程详情页面HTML代码
			course_info_html = self.Requests_html(course_list[j-1])
			#根据详情页HTML返回课程章节信息
				#Chapter[][0]----课程章节名称
				#Chapter[][1]----课程章节对应URL链接
				#Chapter[k]---->>>>>章节名+该章节的URL
			Chapter = self.Chapter_List(course_info_html)
			#课程其他详细信息
				#course_info[0]----课程名称
				#course_info[1]----课程简介
				#course_info[2]----课程标签分类
				#course_info[3]----课程老师
				#course_info[4]----课程老师简介
				#course_info[5]----课程播放次数
				#course_info[6]----课程交流QQ群
				#course_info[7]----课程图片链接
			course_info = self.Course_Info(course_info_html)
			#课程名称--传入Download_All函数，做文件夹名
			course_name = course_info[0]
			#图片链接
			image_url = course_info[7]
			#循环遍历输出当前课程信息
			#for info in course_info:
			#	print info
			print "当前课程全部下载中！"
			down = self.DownLoad_All(Chapter,course_name,course_info,image_url)
			print "下载完了！！！！！---状态码：%s"%down
	#获取网页源代码
	def Requests_html(self,url):
		#自定义请求头
		self.headers = {
					'User-Agent':ua.random,
					"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
					"Accept-Encoding":"gzip, deflate, sdch",
					"Accept-Language":"zh-CN,zh;q=0.8",
					"Cache-Control":"max-age=0",
					"Connection":"keep-alive",
					}
		try:
			req = requests.get(url,headers = self.headers)
			return req
		except :
			return False
	#获取课程章节目录
	def Chapter_List(self,html):
		#课程章节信息：
		#		章节目录----URL链接
		chapter = []
		tree = etree.HTML(html.text)
		name_url = tree.xpath('/html/body/div[5]/div[1]/div/div[1]/ul/li')
		for li in name_url:
			chapter_name = li.xpath('a/span[1]/text()')
			chapter_url = li.xpath('a/@href')
			chapter_name = chapter_name[0]
			chapter_url = 'http://www.maiziedu.com' + chapter_url[0]
			chapters = [chapter_name,chapter_url]
			chapter.append(chapters)
		#返回章节名称+链接列表的列表
		return chapter
	#获取课程相关信息
	def Course_Info(self,html):
		#页面结构化
		tree = etree.HTML(html.text)
		#课程名
		course_name = tree.xpath('/html/body/div[4]/div[2]/h1/text()')
		#课程简介
		course_jianjie = tree.xpath('/html/body/div[4]/div[2]/p[1]/text()')
		#课程标签分类
		course_tag = tree.xpath('/html/body/div[4]/div[2]/p[2]/a/text()')
		#/html/body/div[4]/div[2]/p[2]/a
		if len(course_tag) == 0 :
			course_tag = ["其他"]
		#讲课老师
		course_teacher = tree.xpath('/html/body/div[5]/div[2]/div[1]/div/h3/text()')
		#讲课老师简介
		course_teacher_jianjie = tree.xpath('/html/body/div[5]/div[2]/div[1]/p/text()')
		if len(course_teacher_jianjie) == 0 :
			course_teacher_jianjie = ["无"]
		#课程播放次数
		course_play_times = tree.xpath('/html/body/div[4]/div[2]/p[3]/span[2]/text()')
		#课程交流QQ群
		course_QQ = tree.xpath('/html/body/div[4]/div[2]/p[4]/a[2]/text()')
		course_QQ = ["-*-".join(itertools.chain(course_QQ)).replace(';','')]
		#课程主页image图片链接
		course_image = tree.xpath('/html/body/div[4]/div[1]/img/@src')
		course_image = ['http://www.maiziedu.com' + course_image[0]]

		course_info  = [course_name[0],course_jianjie[0],course_tag[0],\
						course_teacher[0],course_teacher_jianjie[0],\
						course_play_times[0],course_QQ[0],course_image[0]]
		return course_info
	#全部下载当前目录课程章节交互对话框
	def DownLoad_All(self,Chapter,course_name,course_info,image_url):
		print "我在下载当前章节(%s)所有的课程！"%course_name
		try:
			#Chapter[0]----该课程所有的章节播放链接--列表
			for i in range(len(Chapter)):
				down_url = Chapter[i][1]
				print "章节名称：%s"%Chapter[i][0]
				print "链接：%s"%down_url
				path1 = course_name
				#文件夹名--path1
				print path1
				print "*****************"
				path2 = Chapter[i][0]
				#文件名--path2
				print path2
				#传入下载链接--文件夹名--文件名>>>>下载视频
				down1 = self.Download_mp4(down_url,path1,path2)
			#图片链接
			image_url = image_url
			#传入下载链接--文件夹名--文件名>>>>下载图片
			down2 = self.download_Image(image_url,path1,path2)
			#视频下载结束，保存该视频课程简介
			self.Save_Course_Info(course_info,path1,path2)
			return down1
		except :
			return False
	#下载视频
	def Download_mp4(self,url,path1,path2):
		#path1参数为指定文件夹下，课程文件夹名
		#path2参数为指定文件夹下，课程MP4文件名
		print "请求链接：%s"%url
		try:
			html = self.Requests_html(url)
			#使用正则匹配，视频下载链接>>>>>隐藏的视频URL
			com = re.compile(r'lessonUrl = "http://newoss\.maiziedu\.com/.*?\.mp4')
			urls = re.findall(com,html.text)
			down_urls = urls[0][13:]
			print "下载链接：%s"%down_urls
			#创建文件夹，用path参数命名
			path1 = path1.replace('\\','')
			savepath = "E:\\jiaocheng\\F\\Project\\maizi_edu\\Course\\" + path1
			if(os.path.exists(savepath) == False):
				os.makedirs(savepath)
			savepath = savepath + "\\"
			#调用wget模块的download函数，传入参数，下载视频
			wget.download(down_urls,str(savepath) + path2 + '.mp4')
			print "我的视频下载完了哦！"
			time.sleep(3)
			return True
		except :
			return False
	#保存课程相关信息数据
	def Save_Course_Info(self,course_info,path1,path2):
		try:
			#创建文件夹，用path参数命名
			path1 = path1.replace('\\','')
			savepath = "E:\\jiaocheng\\F\\Project\\maizi_edu\\Course\\" + path1
			if(os.path.exists(savepath) == False):
				os.makedirs(savepath)
			savepath = savepath + "\\"
			#将课程信息保存到TXT文本文件中
			with open(str(savepath) + path1 + '.txt','a') as file:
				for info in course_info:
					print info
					file.write(info + '\n')
			file.close()
			return True
		except :
			return False
	#下载当前课程主页图片
	def download_Image(self,image_url,path1,path2):
		print "我正在开始下载图片了哦！"
		try:
			#创建文件夹，用path参数命名
			path1 = path1.replace('\\','')
			savepath = "E:\\jiaocheng\\F\\Project\\maizi_edu\\Course\\" + path1
			if(os.path.exists(savepath) == False):
				os.makedirs(savepath)
			savepath = savepath + "\\"
			#将图片保存为image
			html = self.Requests_html(image_url)
			f = open(str(savepath) + path1 + '.jpg', 'ab')
			f.write(html.content)
			f.close()
			return True
		except :
			return False

def main():
	#实例化一个类，maizi
	maizi = MaiZi()
	#遍历所有课程所有分页链接，总共45页
	for i in range(1,46):
		try:
			#构造课程列表页面链接
			#PythonWeb方向
			#url = 'http://www.maiziedu.com/course/python-all/0-' + str(i) + '/'
			#全部课程
			url = 'http://www.maiziedu.com/course/all-all/0-' + str(i) + '/'
			#请求课程页面HTML代码
			html = maizi.Requests_html(url)
			#调用Course_List函数，获取课程详情页链接
			course_list = maizi.Course_List(html.text)
			#对课程页面url进一步处理
			maizi.Change_Box(course_list)
		except :
			print "SORRY....小哥哥你的电脑网络好像奔溃了~~~~"
			print "确保网络连接正确，重新试一下哦~~~~~~"


if __name__ == '__main__':
	main()